#!/bin/bash
# This a batch submission script for an MVAPICH-Intel job on Trestles
# Submit this script using the command: qsub <script_name>
# Use the "qstat" or "qs" commands to check the status of a job.
# Use the qdel command to delete a job
# qstat -u baden
# qdel <job number>

#
# There are only a few places you'll need to make changes, as marked below
 
# 0. The job queue (normal  or shared)
# 1. The number of nodes  and the number of cores per node
# 2. The wall clock job time limit
# 3. The job name
# 4. An email notification when the job is complete
#   !!! Change to the address you want the notification sent to !!!
# 5. The job

#
# The following are embedded QSUB options.
# The syntax is #PBS followed by option and any arguments, without
# any space between the # and PBS
# The # does _not_  denote that the lines are commented out
# so do not remove it.
# To comment out a line, put a space after the #
#


# The project allocation  (DON'T CHANGE THIS)
#PBS -A csd160

# At the end of the job you will get two files, one each for
# for stdout and stderr, in directory from which qsub was executed
# The filename for stdout is (default = <job_name>.o<job_id>)
# The filename for stderr is (default = <job_name>.e<job_id>)
# Where the <job_name> is specified via the -N option below,
#  and <job_id> is a unique number assigned by the system


# Export all my environment variables to the job
#PBS -V

# ***  --- Changes start here ----


# 0. Specify the batch queue
#
# Use the normal for collecting timings, use the
# shared queue when debugging, as you'll get faster turnaround
#PBS -q normal

#
# 1. Number of nodes and number of processors per node requested
#
# nodes: number of 32-core nodes
#   ppn: how many cores per node to use (1 through 32)
#       (you are always charged for the entire node)
# This value should be 32 for MPI jobs, 1 for threaded ones
#
#
#PBS -l nodes=8:ppn=32

#
# 2. The requested wall clock job time limit in HH:MM:SS
#    Your job will end when it exceeds this time limit
#    Be careful about raising this limit
#
#PBS -l walltime=00:30:00

# filename for standard output (default = <job_name>.o<job_id>)
# at end of job, it will reside in the directory from which qsub was executed
# remove extra ## from the line below if you want to name your own file
###PBS -o job_stdout

# filename for standard error (default = <job_name>.e<job_id>)
# at end of job, it is in the directory from which qsub was executed
# remove extra ## from the line below if you want to name your own file
###PBS -e job_stderr

# 3. Name of the job (default = name of script file)
#PBS -N CANNON-6502-256

#
# 4.  Your email address
#
# Change to the address you want the notification sent to!!
# and REMOVE THE blank between the space and the #
#PBS -M baden@ucsd.edu
#
# Mail is sent when the job terminates normally or abnormally
# Add the letter 'b' if you also want to be notified when the job
# begins to run
#PBS -m aeb
#
#
# **** End of embedded QSUB options
#

# --------- The job starts ---------
# 5.  Put the executeable commands you want to run here
#	It is suggested that you always print out the environment
#	which maybe be useful in troubleshooting
printenv
echo ">>> Job starts"
date
echo ""

# set echo               # echo commands before execution; use for debugging

cd $PBS_O_WORKDIR 	 #change to the working directory

# Put your MPI Commands here
# We  use the  mpirun_rsh command to launch our binaries
# 
#
# mpirun_rsh -np 1 -hostfile $PBS_NODEFILE MV2_ENABLE_AFFINITY=0 ./cannon -n 6502 -i 1
# mpirun_rsh -np 4 -hostfile $PBS_NODEFILE MV2_ENABLE_AFFINITY=0 ./cannon -n 10320 -i 1
# mpirun_rsh -np 16 -hostfile $PBS_NODEFILE MV2_ENABLE_AFFINITY=0 ./cannon -n 16384 -i 4
mpirun_rsh -np 256 -hostfile $PBS_NODEFILE MV2_ENABLE_AFFINITY=0 ./cannon -n 41280 -i 1
mpirun_rsh -np 256 -hostfile $PBS_NODEFILE MV2_ENABLE_AFFINITY=0 ./cannon -n 41280 -i 2


# End of commands
echo ""
date
echo ""
echo ">>> Job Ends"
